package com.bblocks.common.data.support;

import cn.hutool.core.collection.CollUtil;
import cn.hutool.core.util.StrUtil;
import com.bblocks.area.AreaVo;
import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;
import com.google.common.base.Joiner;
import com.google.common.collect.Lists;
import lombok.Data;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import us.codecraft.webmagic.*;
import us.codecraft.webmagic.pipeline.Pipeline;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Selectable;

import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;

import static java.util.regex.Pattern.compile;

/**
 * <p>Project: bblocks-pro - AreaSpider</p>
 * <p>描述：爬取省市区街道社区（5级）-国家统计局
 * 缺陷：此数据比较齐全但是比较杂，并且数据是一年一更可能会存在滞后，需额外移除和处理开发区、经济区、高新区、国家级新区等区域
 * 非直辖师，区级存在 “市辖区”
 * </p>
 *
 * @Author Zhao [125043150@qq.com]
 * @Date 2024/3/15 15:46
 * @Version 1.0
 * @since 8
 */
@Slf4j
public class AreaSpider implements PageProcessor {
    private static final Cache<String, Object> cache = Caffeine.newBuilder()
            .expireAfterWrite(60, TimeUnit.MINUTES)
            .maximumSize(10_000)
            .build();

    /**
     * 公共路径url:
     */
    public static final String SPIDER_URL = "https://www.stats.gov.cn/sj/tjbz/tjyqhdmhcxhfdm/2022/";


    private final Site site = Site.me().setUserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.163 Safari/537.36").setTimeOut(1000 * 60).setCharset("UTF-8").setRetryTimes(10).setSleepTime(100);

    public static final List<String> contains = Lists.newArrayList("city","county","town","village");


    /**
     * 爬取数据的年度标识,用于区分批次
     */
    private int year;

    /**
     * 爬取从省级到包含的那一级数据
     * 例如: contain = "city,county,town,village";  即爬取省级到县级数据
     * 支持：city-城市,county-区/县,town-乡镇/街道,village-社区/村
     */
    private String contain;
    /**
     * 爬取数据等级：0-4
     */
    @Getter
    private int containLv;

    /**
     *
     * @param year
     * @param containLv 0-4,分别采集：省、市、区/县，乡镇/街道,社区/村
     */
    public AreaSpider(int year, int containLv) {
        this.year = year;
        if(containLv >= contains.size())
            containLv = contains.size();

        if(containLv == 0)
            this.contain = "";
        else
            this.contain = Joiner.on(",").join(contains.subList(0,containLv));
        this.containLv = containLv;
    }

    @Override
    public void process(Page page) {
        page.setCharset("UTF-8");
        String url = page.getUrl().toString();
        String[] replaceUrl = url.replace(SPIDER_URL, "").replace(".html", "").split("/");
        if (url.matches(SPIDER_URL + "index.html")) {
            //抓取省级行政区划
            List<AreaVo> provinces = page.getHtml().xpath("//tr[@class='provincetr']/td").nodes().stream()
                    .filter(selectable -> selectable.xpath("//a/text()") != null)
                    .filter(selectable -> selectable.links().all().size() > 0)
                    .map(selectable -> {
                        String name = selectable.xpath("//a/text()").toString();
                        String newUrl = selectable.links().all().get(0);

                        String replace = newUrl.replace(SPIDER_URL, "").replace(".html", "");
                        String areaCode = replace + "0000"; //北京市：110000

                        AreaVo province = AreaVo
                                .builder()
                                .year(year)
                                .lv(1)
                                .name(name)
                                .code(areaCode)
                                .fullName(name)
                                .childUrl(newUrl)
                                .build();

                        if (contain.contains("city")) {
                            addFollowsUrl(page,province,1);
                        }
                        //爬取省级行政区划的时候，爬取的数据只有一条
                        return province;
                    }).collect(Collectors.toList());

            page.putField("area", provinces);
        }

        //抓取市级行政单位
        if (replaceUrl.length == 1 && !replaceUrl[0].equals("index") && contain.contains("city")) {
            List<AreaVo> citys = new ArrayList<>();
            List<Selectable> cityNodes = page.getHtml().xpath("//tr[@class='citytr']/td").nodes();
            cityNodes.stream().forEach(node -> {
                String name = node.xpath("//a/text()").toString();
                if (!compile("[0-9]*").matcher(name).matches()) {
                    String newUrl = node.links().all().get(0);

                    String replace = newUrl.replace(SPIDER_URL, "").replace(".html", "");
                    String[] split = replace.split("/");
                    String parentId = split[0] + "0000";
                    String areaCode = split[split.length - 1] + "00";

                    AreaVo city = AreaVo
                            .builder()
                            .year(year)
                            .lv(2)
                            .name(name)
                            .code(areaCode)
                            .fullName(name)
                            .parentCode(parentId)
                            .childUrl(newUrl)
                            .build();

                    if (contain.contains("county")) {
                        addFollowsUrl(page,city,2);
                    }
                    citys.add(city);
                }
            });
            page.putField("area", citys);
        }

        //抓取县级行政单位
        if (replaceUrl.length == 2 && contain.contains("county")) {
            List<AreaVo> countys = new ArrayList<>();
            List<Selectable> countyNodes = page.getHtml().xpath("//tr[@class='countytr']/td").nodes();
            for (int i = 0; i < countyNodes.size(); i += 2) {
                List<String> code = countyNodes.get(i).xpath("//*/text()").all();
                List<String> name = countyNodes.get(i + 1).xpath("//*/text()").all();
                String areaCode = code.get(0);
                String areaName = name.get(0);
                String newUrl = null;
                if (code.size() > 1) {
                    areaCode = code.get(1);
                    areaName = name.get(1);
                    newUrl = countyNodes.get(i).links().all().get(0);
                }
                areaCode = areaCode.substring(0, 6);
                String parentId = areaCode.substring(0, 4) + "00";

                AreaVo county = AreaVo
                        .builder()
                        .year(year)
                        .lv(3)
                        .name(areaName)
                        .code(areaCode)
                        .fullName(areaName)
                        .parentCode(parentId)
                        .childUrl(newUrl)
                        .build();

                if (contain.contains("town")) {
                    addFollowsUrl(page,county,3);
                }
                countys.add(county);
            }
            page.putField("area", countys);
        }

        //抓取镇级行政单位
        if (replaceUrl.length == 3 && contain.contains("town")) {
            List<AreaVo> towns = new ArrayList<>();
            List<Selectable> townNodes = page.getHtml().xpath("//tr[@class='towntr']/td").nodes();
            for (int i = 0; i < townNodes.size(); i += 2) {
                List<String> code = townNodes.get(i).xpath("//*/text()").all();
                List<String> name = townNodes.get(i + 1).xpath("//*/text()").all();
                String areaCode = code.get(0);
                String areaName = name.get(0);
                String newUrl = null;
                if (code.size() > 1) {
                    areaCode = code.get(1);
                    areaName = name.get(1);
                    newUrl = townNodes.get(i).links().all().get(0);

                }
                areaCode = areaCode.substring(0, 9);
                String parentId = areaCode.substring(0, 6);

                AreaVo town = AreaVo
                        .builder()
                        .year(year)
                        .lv(4)
                        .name(areaName)
                        .code(areaCode)
                        .fullName(areaName)
                        .parentCode(parentId)
                        .childUrl(newUrl)
                        .build();

                if (contain.contains("village")) {
                    addFollowsUrl(page,town,4);
                }
                towns.add(town);
            }
            page.putField("area", towns);
        }

        //抓取乡级行政单位
        if (replaceUrl.length == 4 && contain.contains("village")) {
            List<AreaVo> villages = new ArrayList<>();
            List<Selectable> countyNodes = page.getHtml().xpath("//tr[@class='villagetr']/td").nodes();
            for (int i = 0; i < countyNodes.size(); i += 3) {
                String areaCode = countyNodes.get(i).xpath("//*/text()").get();
                String areaName = countyNodes.get(i + 2).xpath("//*/text()").get();
                String parentId = areaCode.substring(0, 9);

                AreaVo village = AreaVo
                        .builder()
                        .year(year)
                        .lv(5)
                        .name(areaName)
                        .code(areaCode)
                        .fullName(areaName)
                        .parentCode(parentId)
                        .childState(1)
                        .build();
                villages.add(village);
            }
            page.putField("area", villages);
        }
    }

    @Override
    public Site getSite() {
        return site;
    }


    /**
     * 添加后续采集新地址
     * 用于业务控制，控制部分页面二次采集跳过
     * @param page
     * @param parent childUrl 必须存在
     * @param targetLv newUrl采集对应下属等级
     * @return
     */
    protected boolean addFollowsUrl(Page page, AreaVo parent,int targetLv) {
        String newUrl = parent.getChildUrl();
        page.addTargetRequest(newUrl);
        log.info("即将采集地址:{}" , newUrl);
        return true;
    }




    public static void main(String[] args) {
        //采集全国
        /*Spider.create(new AreaSpider(2022, 4))
                .addUrl(SPIDER_URL + "index.html")
                .addPipeline(new LogPipeline())
                .thread(2)
                .run();*/

        long b = System.currentTimeMillis();
        List<AreaVo> result = Lists.newArrayList();
        LogPipeline logPipeline = new LogPipeline(result);
        Spider.create(new AreaSpider(2022, 1))
                .addUrl(SPIDER_URL + "index.html")
                .addPipeline(logPipeline)
                .thread(2)
                .run();

        log.info("采集到1:{}-{}-{}",logPipeline.getFailNum(),result.size(),(System.currentTimeMillis() - b));

        b = System.currentTimeMillis();
        result.clear();
        logPipeline = new LogPipeline(result);
        Spider.create(new AreaSpider(2022, 1))
                .addUrl(SPIDER_URL + "index.html")
                .addPipeline(logPipeline)
                .thread(2)
                .run();
        log.info("采集到2:{}-{}-{}",logPipeline.getFailNum(),result.size(),(System.currentTimeMillis() - b));


        //采集江苏
       /* Spider.create(new AreaSpider(2022, 4))
                .addUrl(SPIDER_URL + "32.html")
                .addPipeline(new LogPipeline())
                .thread(2)
                .run();*/

        /**
         * 采集南京
         */
        /*Spider.create(new AreaSpider(2022, 4))
                .addUrl(SPIDER_URL + "32/3201.html")
                .addPipeline(new LogPipeline())
                .thread(2)
                .run();*/


    }


    /**
     * 日志输出
     */
    @Data
    public static class LogPipeline implements Pipeline {

        List<AreaVo> result;
        /**
         * 失败次数 0=成功，
         */
        int failNum  ;
        public LogPipeline(List<AreaVo> result ){
            this.result = result;
            this.failNum = 0;
        }

        @Override
        public void process(ResultItems resultItems, Task task) {
            String url = resultItems.getRequest().getUrl();
            List<AreaVo> areas = resultItems.get("area");
            log.info("地区总数：{}" , areas.size());

            if (areas.isEmpty()) {//url采集失败
                log.info(url + " 此页面未爬取数据,请稍后重试!");
                failNum ++;
            } else {
                areas.stream().forEach(a -> {
                    log.info("{}" , a);
                });

                this.result.addAll(areas);
            }
        }
    }
}

